from scrapy.spider import Spider
from scrapy.selector import Selector

from spider.items import SpiderItem
import MySQLdb

class BitpageSpider(Spider):

    name = 'bitpage'
    conn = MySQLdb.connect(host="localhost",user="spider",passwd="Xjk@4047218?",db="spider")
    cursor = conn.cursor() 

    sql = "select url from sp_ba_cstemp order by id asc"
    count = cursor.execute(sql)
    if count == 0: 

        sql = "select url from sp_ba_city order by id asc"
        result = cursor.execute(sql)
        cityList = []

        for row in cursor.fetchall():
            if row[0].split("/")[2].split(".")[0] == "www":
                cityList.append(row[0].split("/")[-2])
            else:
                cityList.append(row[0].split("/")[2].split(".")[0])


        sql = "select url from sp_ba_brand order by id asc"
        branddata = cursor.execute(sql)
        brandList = []

        for r in cursor.fetchall():
            brandList.append(r[0].split("/")[-2])

        listUrl = []

        for i in cityList:
            for j in brandList:
                listUrl.append("http://dealer.bitauto.com/"+i+"/"+j+"/")


        for i in listUrl:
            sql = "insert into sp_ba_cstemp(`url`) values('"+i+"')"
            print sql
            cursor.execute(sql)
            cursor.execute("commit")
        start_urls = listUrl
    else:
        listUrl = []
        for row in cursor.fetchall():
            listUrl.append(row[0])    
        start_urls = listUrl
 
#    print listUrl
#    print len(listUrl)
#    start_urls = ["http://dealer.bitauto.com/baiyin/volkswagen/"]
#    start_urls = ["http://dealer.bitauto.com/beijing/volkswagen/"]
#    start_urls = listUrl

    def parse(self,response):
       sel = Selector(response)
       item = SpiderItem()
       area = sel.xpath('//div[@id="divDealerList"]')
      
       pagearea = area.xpath('.//div[@class="the_pages"]').extract()
       if len(pagearea) != 0:
           temp = area.xpath('.//div[@class="the_pages"]//a/text()').extract()       
           item["page"] = len(temp)-1
           item["type"] = "bitpage"
           item["carshopurl"] = response.url
           return item
       else:
           print "Nothing !"
           pass





